# -*- coding: utf-8 -*-
# MailArchive plugin

import cmd
import sys

import urllib
import time
import calendar
import re
import os
import tempfile
import email.Errors
import email.Utils
import mailbox
import mimetypes
import email
#from email.Parser  import Parser
from email.Header import decode_header
#from email.Utils import collapse_rfc2231_value
from email.Utils import unquote
import traceback

#011
import pkg_resources

from genshi.builder import tag

from genshi.core import Stream, Markup as GenshiMarkup
from genshi.input import HTMLParser, ParseError, HTML

from trac.core import *
from trac.env import IEnvironmentSetupParticipant
#from trac.Search import ISearchSource, search_to_sql, shorten_result
from trac.search import ISearchSource, search_to_sql, shorten_result

from trac.ticket.model import Ticket

from trac.web import IRequestHandler
from trac.util import NaivePopen
from StringIO import StringIO

import poplib

from trac.wiki import wiki_to_html,wiki_to_oneliner, IWikiSyntaxProvider
from trac.wiki.api import WikiSystem
from trac.util.html import html, Markup #0.10
from trac.web.chrome import add_link, add_stylesheet, INavigationContributor, ITemplateProvider

#0.11 from trac.attachment import attachment_to_hdf, attachments_to_hdf, Attachment, AttachmentModule
from trac.attachment import Attachment,AttachmentModule




from trac.mimeview import *
#from trac.mimeview.api import Mimeview, IContentConverter #0.10


from trac.admin.console import TracAdmin

from api import IEmailHandler
from util import *

class MailArchiveAdmin(TracAdmin):
    intro = ''
    doc_header = 'Trac MailArchive Plugin Admin Console\n'
    
    ruler = ''
    prompt = "MailArchiveAdmin> "
    __env = None
    _date_format = '%Y-%m-%d'
    _datetime_format = '%Y-%m-%d %H:%M:%S'
    _date_format_hint = 'YYYY-MM-DD'

    ## Help
    _help_help = [('help', 'Show documentation')]

    def all_docs(cls):
        return (cls._help_help)
    all_docs = classmethod(all_docs)



    def msgfactory(self,fp):
        try:
            return email.message_from_file(fp)
        except email.Errors.MessageParseError:
            # Don't return None since that will
	    # stop the mailbox iterator
	    return ''

    def decode_to_unicode(self, basestr):
        # http://www.python.jp/pipermail/python-ml-jp/2004-June/002932.html
        # Make mail header string to unicode string

        decodefrag = email.Header.decode_header(basestr)
        subj_fragments = ['',]
        for frag, enc in decodefrag:
            if enc:
                frag = self.to_unicode(frag, enc)
            subj_fragments.append(frag)
        return ''.join(subj_fragments)

    def to_unicode(self,text,charset):
        if text=='':
            return ''

        default_charset = self.env.config.get('mailarchive', 'default_charset',None)
        if default_charset :
            chaerset = default_charset

        # to unicode with codecaliases
        # codecaliases change mail charset to python charset
        charset = charset.lower( )
        aliases = {}
        aliases_text = self.env.config.get('mailarchive', 'codecaliases')
        for alias in aliases_text.split(','):
            alias_s = alias.split(':')
            if len(alias_s) >=2:
                if alias_s[1] == 'cmd':
                    aliases[alias_s[0].lower()] = ('cmd',alias_s[2])
                else:
                    aliases[alias_s[0].lower()] = ('codec',alias_s[1])

        if aliases.has_key(charset):
            (type,alias) = aliases[charset]
            if type == 'codec':
                text = unicode(text,alias)
            elif type == 'cmd':
                np = NaivePopen(alias, text, capturestderr=1)
                if np.errorlevel or np.err:
                    err = 'Running (%s) failed: %s, %s.' % (alias, np.errorlevel,
                                                            np.err)
                    print err
                    raise Exception, err
                text = unicode(np.out,'utf-8')
        else:
            text = unicode(text,charset)
        return text

    def import_message(self, msg, author,mlid, db):
        OUTPUT_ENCODING = 'utf-8'
        subject = ''
        messageid = ''
        utcdate = 0
        localdate = 0
        zoneoffset = 0
        text = ''
        fromtext = ''
        body = ''
        ref_messageid = ''

        cursor = db.cursor()
        is_newid = False

        if 'message-id' in msg:
            messageid = msg['message-id']
            if messageid[:1] == '<':
                messageid = messageid[1:]
            if messageid[-1:] == '>':
                messageid = messageid[:-1]
            self.print_debug('Message-ID:%s' % messageid )

            #check messageid is unique
            self.print_debug("Creating new mailarc '%s'" % 'mailarc')
            cursor.execute("SELECT id from mailarc WHERE messageid=%s",(messageid,))
            row = cursor.fetchone()
            id = None
            if row:
                id = row[0]
            if id == None or id == "":
                # why? get_last_id return 0 at first.
                #id = db.get_last_id(cursor, 'mailarc')
                is_newid = True
                cursor.execute("SELECT Max(id)+1 as id from mailarc")
                row = cursor.fetchone()
                if row and row[0] != None:
                    id = row[0]
                else:
                    id = 1
            id = int(id) # Because id might be 'n.0', int() is called.


        if 'date' in msg:
            datetuple_tz = email.Utils.parsedate_tz(msg['date'])
            localdate = calendar.timegm(datetuple_tz[:9]) #toDB
            zoneoffset = datetuple_tz[9] # toDB
            utcdate = localdate-zoneoffset # toDB
            #make zone ( +HHMM or -HHMM
            zone = ''
            if zoneoffset >0:
                zone = '+' + time.strftime('%H%M',time.gmtime(zoneoffset))
            elif zoneoffset < 0:
                zone = '-' + time.strftime('%H%M',time.gmtime(-1*zoneoffset))

            #self.print_debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9]))
            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(utcdate)))
            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate)))
            self.print_debug(zone)

        fromname,fromaddr = email.Utils.parseaddr(msg['from'])
        fromname = self.decode_to_unicode(fromname)
        fromaddr = self.decode_to_unicode(fromaddr)

        self.print_info( '  ' + time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))+' ' + zone +' '+ fromaddr)

        if 'subject' in msg:
            subject = self.decode_to_unicode(msg['subject'])
            self.print_debug( subject.encode(OUTPUT_ENCODING))

        # make thread infomations
        ref_messageid = ''
        if 'in-reply-to' in msg:
            ref_messageid = ref_messageid + msg['In-Reply-To'] + ' '
            self.print_debug('In-Reply-To:%s' % ref_messageid )

        if 'references' in msg:
            ref_messageid = ref_messageid + msg['References'] + ' '

        m = re.findall(r'<(.+?)>', ref_messageid)
        ref_messageid = ''
        for text in m:
            ref_messageid = ref_messageid + "'%s'," % text
        ref_messageid = ref_messageid.strip(',')
        self.print_debug('RefMessage-ID:%s' % ref_messageid )


        # multipart mail
        if msg.is_multipart():
            body = ''
            # delete all attachement at message-id
            Attachment.delete_all(self.env, 'mailarchive', id, db)

            for part in msg.walk():
                content_type = part.get_content_type()
                self.print_debug('Content-Type:'+content_type)
                file_counter = 1

                if content_type == 'multipart/mixed':
                    pass
                elif content_type == 'text/html' and self.is_file(part) == False:
                    body = part.get_payload(decode=1)
                elif content_type == 'text/plain' and self.is_file(part) == False:
                    body = part.get_payload(decode=1)
                    charset = part.get_content_charset()
                    self.print_debug('charset:'+str(charset))
                    # Todo:need try
                    if charset != None:
                        body = self.to_unicode(body,charset)
                elif part.get_payload(decode=1) == None:
                    pass
                else:
                    self.print_debug( part.get_content_type())
                    # get filename
                    # Applications should really sanitize the given filename so that an
                    # email message can't be used to overwrite important files
                    filename = self.get_filename(part)
                    if not filename:
                        ext = mimetypes.guess_extension(part.get_content_type())
                        if not ext:
                            # Use a generic bag-of-bits extension
                            ext = '.bin'
                        filename = 'part-%03d%s' % (file_counter, ext)
                        file_counter += 1

                    self.print_debug("filename:" + filename.encode(OUTPUT_ENCODING))

                    # make attachment
                    tmp = os.tmpfile()
                    tempsize =len(part.get_payload(decode=1))
                    tmp.write(part.get_payload(decode=1))

                    tmp.flush()
                    tmp.seek(0,0)

                    attachment = Attachment(self.env,'mailarchive', id)

                    attachment.description = '' # req.args.get('description', '')
                    attachment.author = author #req.args.get('author', '')
                    attachment.ipnr = '127.0.0.1'

                    try:
                        attachment.insert(filename,
                                tmp, tempsize,None,db)
                    except Exception, e:
                        try:
                            ext = filename.split('.')[-1]
                            if ext == filename:
                                ext = '.bin'
                            else:
                                ext = '.' + ext
                            filename = 'part-%03d%s' % (file_counter, ext)
                            file_counter += 1
                            attachment.insert(filename,
                                    tmp, tempsize,None,db)
                            self.print_warning('As name is too long, the attached file is renamed : '+filename)

                        except Exception, e:
                            self.print_error('Exception at attach file of Message-ID:'+messageid)
                            self.print_error( e )

                    tmp.close()

        # not multipart mail
        else:
            # Todo:if Content-Type = text/html then convert htmlMail to text
            content_type = msg.get_content_type()
            self.print_debug('Content-Type:'+content_type)
            if content_type == 'text/html':
                body = msg.get_payload(decode=1)
                charset = msg.get_content_charset()
     
                # need try:
                if charset != None:
                    self.print_debug("charset:"+charset)
                    body = self.to_unicode(body,charset)

                body = unicode(body)
                
                from stripogram import html2text, html2safehtml
                body = html2text(body)

            else:
                #body
                #self.print_debug(msg.get_content_type())
                body = msg.get_payload(decode=1)
                charset = msg.get_content_charset()

                # need try:
                if charset != None:
                    self.print_debug("charset:"+charset)
                    body = self.to_unicode(body,charset)


        #body = body.replace(os.linesep,'\n')
        self.print_debug('Thread')

        thread_parent = ref_messageid.replace("'",'').replace(',',' ')
        thread_root = ''
        if thread_parent !='':
        # sarch first parent id
            self.print_debug("SearchThread;"+thread_parent)
            cursor = db.cursor()
            sql = "SELECT threadroot,messageid FROM mailarc where messageid in (%s)" % ref_messageid
            self.print_debug(sql)
            cursor.execute(sql)

            row = cursor.fetchone()
            if row:
                #thread_parent = row[1]
                if row[0] == '':
                    thread_root = thread_parent.split(' ').pop()
                    self.print_debug("AddToThread;"+thread_root)
                else:
                    thread_root = row[0]
                    self.print_debug("NewThread;"+thread_root)
            else:
                    self.print_debug("NoThread;"+thread_parent)
        thread_root = thread_root.strip()

        self.print_debug('Insert')

        if messageid != '':

            # insert or update  mailarc_category

            yearmonth = time.strftime("%Y%m",time.gmtime(utcdate))
            category = mlid+yearmonth
            cursor.execute("SELECT category,mlid,yearmonth,count FROM mailarc_category WHERE category=%s",(category.encode('utf-8'),))
            row = cursor.fetchone()
            count = 0
            if row:
                count = row[3]
                pass
            else:
                cursor.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category.encode('utf-8'),mlid.encode('utf-8'),yearmonth,0))
            if is_newid == True:
                count = count +1
            cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s" ,
                (count,category.encode('utf-8')))

            # insert or update mailarc

            #self.print_debug(
            #    "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id),
            #    category.encode('utf-8'),
            #    messageid,
            #     utcdate,
            #      zoneoffset,
            #     subject.encode('utf-8'), fromname.encode('utf-8'),
            #     fromaddr.encode('utf-8'),'','',
            #     thread_root,thread_parent))
            cursor.execute("DELETE FROM mailarc where messageid=%s",(messageid,))
            #print is_newid
            cursor.execute("INSERT INTO mailarc ("
                "id,category,messageid,utcdate,zoneoffset,subject,"
                "fromname,fromaddr,header,text, threadroot,threadparent ) "
                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
                (str(id),
                category.encode('utf-8'),
                messageid,
                 utcdate,
                  zoneoffset,
                 subject.encode('utf-8'), fromname.encode('utf-8'),
                 fromaddr.encode('utf-8'),'',body.encode('utf-8'),
                 thread_root,thread_parent))

        db.commit()
        # after commit process...
        #print "From: %s, Subject: %s" %(fromaddr.encode('utf-8'), subject.encode('utf-8'))
        
        message = {"id":id, "messageid":messageid, "category":category.encode('utf-8'), 
                   "utcdate":utcdate, "zoneoffset":zoneoffset,
                   "subject":unicode(subject.encode('utf-8')),
                   "fromname":unicode(fromname.encode('utf-8')),
                   "fromaddr":unicode(fromaddr.encode('utf-8')),
                   "body":unicode(body.encode('utf-8')),
                   "thread_root":thread_root,
                   "thread_parent":thread_parent
                   }

        self.handleMail(is_newid, message)

    def handleMail(self, is_newid, message):
        # handle the message
        handlers = ExtensionPoint(IEmailHandler).extensions(self.env)
        
        handlers.sort(key=lambda x: x.order(), reverse=True)
        warnings = []
        for handler in handlers:
            try:
                if is_newid:
                    message = handler.invoke(IEmailHandler.INSERT, message, warnings)
                else:
                    message = handler.invoke(IEmailHandler.UPDATE, message, warnings)
            except Exception, e:
                traceback.print_exc(e)
                self.print_error('Subject: %s\n\n%s' % (str(message['subject']), str(e)))
                raise e
        
    def do_refresh_category(self,line):
        db = self.db_open()
        self.env = self.env_open()
        cursor = db.cursor()
        cursor.execute("DELETE FROM mailarc_category")
        cursor.execute("SELECT category, count(*) as cnt from mailarc GROUP BY category ")
        for category,cnt in cursor:
            cursor2 =  db.cursor()
            cursor2.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category,category[:-6],category[-6:],cnt))
        db.commit()

    ## Help
    _help_import = [('import <mlname> <filepath>', 'import UnixMail')]

    def do_import(self,line):
        arg = self.arg_tokenize(line)
        if len(arg) < 2 :
            print "import MLname filepath"
            return
        db = self.db_open()
        self.env = self.env_open()
        self._import_unixmailbox('cmd',db,arg[0],arg[1])

    ## Help
    _help_pop3 = [('pop3 <mlname>', 'import from pop3 server')]

    def do_pop3(self,line):
        arg = self.arg_tokenize(line)
        if len(arg) < 1 :
            print "pop3 MLname"
        db = self.db_open()
        self.env = self.env_open()
        self._import_from_pop3('cmd',db,arg[0])

    ## Help
    _help_help = [('help', 'Show documentation')]

    def do_help(self, line=None):
        arg = self.arg_tokenize(line)
        if arg[0]:
            try:
                doc = getattr(self, "_help_" + arg[0])
                self.print_doc (doc)
            except AttributeError:
                print "No documentation found for '%s'" % arg[0]
        else:
            docs = (#self._help_about +
                    self._help_help +
                    self._help_import + self._help_pop3
                    )
            print 'mailarc-admin - The Trac MailArchivePlugin Administration Console '
            if not self.interactive:
                print
                print "Usage: mailarc-admin </path/to/projenv> [command [subcommand] [option ...]]\n"
                print "Invoking mailarc-admin without command starts "\
                      "interactive mode."
            self.print_doc (docs)



    def print_info(self,line):
        print "%s" % line

    def print_debug(self,line):
        #print "[Debug] %s" % line
        pass

    def print_error(self,line):
        print "[Error] %s" % line

    def print_warning(self,line):
        print "[Warning] %s" % line

    def _import_unixmailbox(self,author, db, mlid, msgfile_path):
        self.print_debug('import_mail')
        if not db:
            #db = self.env.get_db_cnx()
            handle_ta = True
        else:
            handle_ta = False


        #paser = Parser()

        self.print_info("%s Start Importing %s ..." %
            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),msgfile_path))

        fp = open(msgfile_path,"rb")
        mbox = mailbox.UnixMailbox(fp, self.msgfactory)

        counter =1
        msg = mbox.next()
        while msg is not None:
            messageid = ''
            try:
                messageid = msg['message-id']
                self.import_message(msg,author,mlid,db)
            except Exception, e:
                traceback.print_exc(e)
                exception_flag = True
                self.print_error('Exception At Message-ID:'+messageid)

            if counter > 10000:
                break
            msg = mbox.next()
            counter = counter + 1


        fp.close()
        #if handle_ta:
        db.commit()
        self.print_info("End Imporing %s. " % msgfile_path)

    def _import_from_pop3(self,author, db, mlid):

#        pop_server = self.env.config.get('mailarchive', 'pop3_server')
#        pop_user = self.env.config.get('mailarchive', 'pop3_user')
#        pop_password = self.env.config.get('mailarchive', 'pop3_password')
#        pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none')

        pop_server = self.env.config.get('mailarchive', 'pop3_server_'+mlid)
        if pop_server =='':
            pop_server = self.env.config.get('mailarchive', 'pop3_server')

        pop_user = self.env.config.get('mailarchive', 'pop3_user_'+mlid)
        if pop_user =='':
            pop_user = self.env.config.get('mailarchive', 'pop3_user')

        pop_password = self.env.config.get('mailarchive', 'pop3_password_'+mlid)
        if pop_password =='':
            pop_password = self.env.config.get('mailarchive', 'pop3_password')

        pop_delete = self.env.config.get('mailarchive', 'pop3_delete_'+mlid)
        if pop_delete =='':
            pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none')

        if pop_server =='':
            self.print_error('trac.ini mailarchive pop3_server is null!')
        elif pop_user == '':
            self.print_error('trac.ini mailarchive pop3_user is null!')
        elif pop_password == '':
            self.print_error('trac.ini mailarchive pop3_password is null!')

        self.print_info("%s Start Connction pop3 %s:%s ..." %
            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),
            pop_server,pop_user))

        pop = poplib.POP3(pop_server)
        pop.user(pop_user)
        pop.pass_(pop_password)
        num_messages = len(pop.list()[1])
        counter = 1
        for i in range(num_messages):
            #lines = ['',]
            #for j in pop.retr(i+1)[1]:
            #    lines.append(j + os.linesep)
            #mes_text = ''.join(lines)
            mes_text = ''.join(['%s\n' % line for line in  pop.retr(i+1)[1]])
            messageid = ''
            exception_flag = False
            try:
                msg = email.message_from_string(mes_text)
                messageid = msg['message-id']
                self.import_message(msg,author,mlid,db)
            except Exception, e:
                traceback.print_exc(e)
                exception_flag = True
                self.print_error('Exception At Message-ID:'+messageid)

            #if exception_flag == False:
            #    self.print_info("    Import Message Success")


            # delete mail
            if pop_delete == 'all':
                pop.dele(i+1)
                self.print_info("    Delete MailServer Message ")
            elif pop_delete == 'imported':
                if exception_flag == False:
                    pop.dele(i+1)
                    self.print_info("    Delete MailServer Message ")
            else:
                pass

            if counter > 10000:
                break
            counter = counter + 1

        pop.quit()

        #if handle_ta:
        db.commit()
        self.print_info("End Reciving. "  )

    def is_file(self,part ):
        """Return True:filename associated with the payload if present.
        """
        missing = object()
        filename = part.get_param('filename', missing, 'content-disposition')
        if filename is missing:
            filename = part.get_param('name', missing, 'content-disposition')
        if filename is missing:
            return False
        return True

    def get_filename(self,part , failobj=None):
        """Return the filename associated with the payload if present.

        The filename is extracted from the Content-Disposition header's
        `filename' parameter, and it is unquoted.  If that header is missing
        the `filename' parameter, this method falls back to looking for the
        `name' parameter.
        """
        missing = object()
        filename = part.get_param('filename', missing, 'content-disposition')
        if filename is missing:
            filename = part.get_param('name', missing, 'content-disposition')
        if filename is missing:
            return failobj

        errors='replace'
        fallback_charset='us-ascii'
        if isinstance(filename, tuple):
            rawval = unquote(filename[2])
            charset = filename[0] or 'us-ascii'
            try:
                return self.to_unicode(rawval, charset)
            except LookupError:
                # XXX charset is unknown to Python.
                return unicode(rawval, fallback_charset, errors)
        else:
            return self.decode_to_unicode(unquote(filename))


def run(args=None):
    """Main entry point."""
    if args is None:
        args = sys.argv[1:]
    admin = MailArchiveAdmin()
    if len(args) > 0:
        if args[0] in ('-h', '--help', 'help'):
            return admin.onecmd('help')
        elif args[0] in ('-v','--version'):
            print '%s %s' % (os.path.basename(args[0]), TRAC_VERSION)
        else:
            admin.env_set(os.path.abspath(args[0]))
            if len(args) > 1:
                s_args = ' '.join(["'%s'" % c for c in args[2:]])
                command = args[1] + ' ' +s_args
                return admin.onecmd(command)
            else:
                while True:
                    admin.run()
    else:
        return admin.onecmd("help")
    
if __name__ =='__main__':
    run()
